//var citiesList = "chongqing";

//var cities = citiesList.split(",");
var cities = []
process.argv.forEach(function (val, index, array) {
    if (index >= 2)
        cities.push(val)
});

var path = require('path'), request = require('request'), cheerio = require('cheerio');
var MongoClient = require('mongodb').MongoClient, assert = require('assert');

var Crawler = require('../../lib/crawler');
//var connection = 'mongodb://localhost:27017/66xiaoqu';
//var connection = 'mongodb://192.168.1.123:27017/66xiaoqu';
var connection = 'mongodb://120.55.126.201:27017/Communities';

var async = require('async');

var dbconn = null;

cities.forEach(function (city) {
// async.eachSeries(cities, function(city){
    var url = 'http://' + city + '.anjuke.com/community/view/';
    var dbcollection = 'xq_anjuke_' + city;
    var dbcollectionDetail = 'xq_anjukeDetail_' + city;
    //var dbcollection = 'xq_' + city;
    //var dbcollectionDetail = 'xq_' + city + '_detail';

    var totalData = 0
    var dataCount = 0  // stores the existing count in the detail collection so that the crawler will skip over the existing data. Relies on Mongo collecting data in ascending order everytime
    var counter = 0
    var db = null;

    function getExistingDataCount(callback) {
        MongoClient.connect(connection,
            function (err, db) {
                assert.equal(null, err);
                //console.log("Connected correctly to server");
                var coll = db.collection(dbcollection);
                coll.count(function (err, count) {
                    totalData = count
                    console.log("Total data %s: %s", dbcollection, totalData)
                    var coll = db.collection(dbcollectionDetail);
                    coll.count(function (err, count) {
                        dataCount = count
                        console.log("Data already in %s: %s", dbcollectionDetail, dataCount)
                        callback();
                    });
                });
            });
    };


    function run() {
        var manifest = {
            /*
             target: {
             url: 'http://shanghai.anjuke.com/community/view/',
             ranges: {
             // number: [1, 1976, 1] // from 0 to 100 and step 5 numbers each time
             // letter: ['a', 'z'] // from a to z, put {{letter}} where you want it
             // resource: 'csv' // configure this resource and put {{resource:csv}}
             resource: 'mongodb' // configure it and put {{resource:mongodb}}
             },
             interval: 500 // Wait 0.5 seconds between each access
             },

             target: {
             url: 'http://beijing.anjuke.com/community/view/',
             ranges: {
             // number: [1, 1976, 1] // from 0 to 100 and step 5 numbers each time
             // letter: ['a', 'z'] // from a to z, put {{letter}} where you want it
             // resource: 'csv' // configure this resource and put {{resource:csv}}
             resource: 'mongodb' // configure it and put {{resource:mongodb}}
             },
             interval: 500 // Wait 0.5 seconds between each access
             },

             target: {
             url: 'http://guangzhou.anjuke.com/community/view/',
             ranges: {
             // number: [1, 1976, 1] // from 0 to 100 and step 5 numbers each time
             // letter: ['a', 'z'] // from a to z, put {{letter}} where you want it
             // resource: 'csv' // configure this resource and put {{resource:csv}}
             resource: 'mongodb' // configure it and put {{resource:mongodb}}
             },
             interval: 500 // Wait 0.5 seconds between each access
             },
             */

            target: {
                url: url,
                ranges: {
                    // number: [1, 1976, 1], // from 0 to 100 and step 5 numbers each time
                    // letter: ['a', 'z'] // from a to z, put {{letter}} where you want it
                    // resource: 'csv' // configure this resource and put {{resource:csv}}
                    resource: 'mongodb' // configure it and put {{resource:mongodb}}
                },
                interval: 1000 // Wait 0.5 seconds between each access
            },
//            resources: {
//                mongodb: {
//                    plugin: 'mongodb',
//                    options: {
////                collection: 'xq_shanghai',
//                        collection: dbcollection,
//                        field: 'commId'
//                    }
//                }
//            },
//
//            plugins: {
//                mongodb: {
//                    connection: connection
//                },
//                csv: true
//            },

            extractors: [{
                name: 'details',
                //root: '//div[@class="border-info comm-detail"]',
                root: '//div[@class="border-info comm-basic clearfix"]',
                fields: {
                    name: '//div[@class="comm-list clearfix"]/dl[@class="comm-l-detail float-l"]/dd[1]/text()',
                    district: '//div[@class="comm-list clearfix"]/dl[@class="comm-l-detail float-l"]/dd[2]/a[1]/text()',
                    address: '//div[@class="comm-list clearfix"]/dl[@class="comm-l-detail float-l"]/dd[3]/em[1]/text()',
                    developedBy: '//div[@class="comm-list clearfix"]/dl[@class="comm-l-detail float-l"]/dd[4]/text()',
                    managedBy: '//div[@class="comm-list clearfix"]/dl[@class="comm-l-detail float-l"]/dd[5]/text()',
                    propertyType: '//div[@class="comm-list clearfix"]/dl[@class="comm-l-detail float-l"]/dd[6]/text()',
                    fees: '//div[@class="comm-list clearfix"]/dl[@class="comm-l-detail float-l"]/dd[7]/text()',
                    constructionArea: '//div[@class="comm-list clearfix"]/dl[@class="comm-r-detail float-r"]/dd[1]/text()',
                    units: '//div[@class="comm-list clearfix"]/dl[@class="comm-r-detail float-r"]/dd[2]/text()',
                    dateBuilt: '//div[@class="comm-list clearfix"]/dl[@class="comm-r-detail float-r"]/dd[3]/text()',
                    rjl: '//div[@class="comm-list clearfix"]/dl[@class="comm-r-detail float-r"]/dd[4]/text()',
                    rentalPercentage: '//div[@class="comm-list clearfix"]/dl[@class="comm-r-detail float-r"]/dd[5]/text()',
                    parkingUnits: '//div[@class="comm-list clearfix"]/dl[@class="comm-r-detail float-r"]/dd[6]/text()',
                    greenPercentage: '//div[@class="comm-list clearfix"]/dl[@class="comm-r-detail float-r"]/dd[7]/text()',
                    desc: '//div[@class="comm-description"]/div[@class="desc-cont"]/text()',

                    subdistrict: '//div[@class="comm-list clearfix"]/dl[@class="comm-l-detail float-l"]/dd[2]/a[2]/text()',

                }
            }]
        };

        var dbconn = null;

        var insertDocuments = function (doc, callback) {
            var collection = dbconn.collection(dbcollectionDetail);

            /*
             var regex = /^http:(.*)#l1=(.*)&l2=(.*)&l3=(.*)&(.*)&commname=(.*)&commid=(.*)(&?)/;
             var result = a.map.match(regex);
             var lat = result[2];
             var lng = result[3];
             var commname = result[6];
             var commid = result[7];
             */
            collection.insert(
                [
                    //{
                    //    commId: doc.commId,
                    //    name : doc.name,
                    //    district: doc.district.trim(),
                    //    subdistrict: doc.subdistrict.trim(),
                    //    address: doc.address,
                    //    developedBy: doc.developedBy,
                    //    managedBy: doc.managedBy,
                    //    propertyType: doc.propertyType,
                    //    fees: doc.fees,
                    //    constructionArea: doc.constructionArea,
                    //    units: doc.units,
                    //    dateBuilt: doc.dateBuilt,
                    //    rjl: doc.rjl,
                    //    parkingUnits: doc.parkingUnits,
                    //    greenPercentage: doc.greenPercentage,
                    //    rentalPercentage: doc.rentalPercentage,
                    //    desc: doc.desc,
                    //    url: doc.url
                    //}
                    doc
                ]
                , function (err, result) {
                    //assert.equal(err, null);
                    //assert.equal(1, result.result.n);
                    //assert.equal(1, result.ops.length);
                    console.log("Inserted %s into the document collection", doc.name);
                    callback(result);
                });
        };

        var job = new Crawler(manifest);

        job.on('data', function (data) {
            console.log(data.data);

            data.data.forEach(
                function (a) {
//                   insertDocuments(a, function(){});
                })
        });

        job.on('error', function (err) {
            throw error;
        });

        job.on('end', function () {
            console.log('The job is done');
            db.close()
            // not exactly,
        });

        var findCommids = function (db, callback) {
//    var collection = db.collection("xq_shanghai");
//    var collection = db.collection("xq_beijing");
//    var collection = db.collection("xq_guangzhou");
            var collection = db.collection(dbcollection);
            collection.find({}).toArray(
                function (err, docs) {
//            console.log(docs);

                    callback(docs);
                });
        };

        var ids = [];

        function init(callback) {
            MongoClient.connect(connection,
                function (err, db) {
                    assert.equal(null, err);
                    console.log("Connected correctly to server");
                    dbconn = db;
                    callback(null, 'init');
                });

        }

        function getIds(callback) {
            findCommids(dbconn, function (a) {
                    ids = a;
                    callback(null, 'getIds');
                }
            );
        }

        function start(callback) {
            console.log("starting...");
            //var url = manifest.target.url;
            var cnt = 0;

            async.eachSeries(ids, function (item, callbackInner) {
                    console.log("(%s: %s of %s)",city,counter,totalData)
                    counter++
                    if (counter - 1 < dataCount) {
                        callbackInner()
                        return
                    }

                    manifest.target.url = item.href
                    //url + item.commId;
                    console.log(manifest.target.url);

                    var job1 = new Crawler(manifest);

                    job1.on('data', function (data) {
                        var $ = cheerio.load(data.request.body);

                        var houseData = $('div.comm-infos.float-r')[0].children[1].children[1]
                        var commData = $('div.border-info.comm-detail')[0].children[1]

                        var commDataL = commData.children[1]
                        var commDataR = commData.children[3]

                        var doc = {}

                        var city = $('div.p_1180.p_crumbs')[0].children[3].children[0].data.replace('小区', '')
                        doc.city = city

                        try {

                            var tagData = houseData.children[7].children
                            var tags = []
                            var tLen = tagData.length
                            for (var i = 0; i < tLen; i++) {
                                if ('name' in tagData[i] && tagData[i].name == 'a')
                                    tags.push(tagData[i].children[0].data)
                            }
                            doc.tags = tags

                            var districtData = commDataL.children[6]
                            doc.district = districtData.children[0].children[0].data.trim().replace('小区', '').replace('新区', '').replace('区', '')
                            doc.subdistrict = districtData.children[2].children[0].data.trim().replace('小区', '').replace('新区', '').replace('区', '')
                            doc.developer = commDataL.children[12].children[0].data
                            doc.manager = commDataL.children[15].children[0].data
                            doc.propertyType = commDataL.children[18].children[0].data
                            doc.fees = commDataL.children[21].children[0].data

                            doc.area = commDataR.children[2].children[0].data
                            doc.units = commDataR.children[5].children[0].data

                            var dateBuildStr = commDataR.children[8].children[0].data.split('-')
                            var year = dateBuildStr[0]
                            var month = dateBuildStr[1]
                            doc.dateBuilt = new Date(year, month)
                            doc.volume = commDataR.children[11].children[0].data
                            //doc.rate = commDataR.children[14].children[0].data    // this data is unavailable without rendering
                            doc.parkingUnits = parseInt(commDataR.children[17].children[0].data.replace(/[^0-9\.]+/g, ''))
                            doc.greenPercentage = parseInt(commDataR.children[20].children[0].data.replace(/[^0-9\.]+/g, ''))
                        } catch (e) {
                            console.log(e)
                        }

                        // copying all fields from the non-detail docs ensures _id field replication, which prevents duplicates upon multiple reinserts and enables easy cross-checking between the detail and non-detail collections
                        for (var field in item) {
                            doc[field] = item[field]
                        }
                        doc.dateCreated = new Date()

                        insertDocuments(doc, function () {
                        })
                    });

                    job1.on('error', function (err) {
                        throw error;
                    });

                    job1.on('end', function () {

                        console.log('The job is done: %d', ++cnt);
                        delete job1;

                        callbackInner();
                    });

                    job1.start();

                }, function (err) {
                    if (err) {
                        console.log('err');
                    }

                }
            );
        }

        function finish(callback) {
            console.log("closing db...");
//    dbconn.close();
            callback(null, 'end');
        }

        // Start our job
        async.series([
                init,
                getIds,
                start,
                finish
            ]
            , function (err, results) {
                console.log(results);

            });
    }

    getExistingDataCount(run);
});

/*
 node /workspace/CrawlerJS/examples/cities/anjuke-master-detail.js

 shanghai
 beijing guangzhou shenzhen chongqing
 hangzhou suzhou chengdu changsha nanjing

 tianjin
 */
